package com.gxljc.bear.crawler.itaogao.caijing;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;

public class CJYuanChuangDetailCrawler extends NewsBaseCrawler {

	/**
	 *
	 */
	private static final long serialVersionUID = 2478822660849227039L;
	private static final Logger LOG = Logger.getLogger(CJYuanChuangDetailCrawler.class);
	private static final String NEWS_LIST_FORMAT = "http://yuanchuang.caijing.com.cn/%s.shtml";
	private static final int MAX_SIZE = 50;
	private static final String YUANCHUANG_FALG = "yuanchuang";

	public CJYuanChuangDetailCrawler() {
		super("t_site_caijing", DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
	}

	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genSeeds() {
		LOG.info("--- genSeed of CaiJing YuanChuang----");
		List<String> ids = new ArrayList<>();
		try {

			for (int i = 1; i <= MAX_SIZE; i++) {
				// fetch url list by parser
				String url = String.format(NEWS_LIST_FORMAT, i + "");

				String result = "";
				Pair<String, byte[]> pair = fetResult(url, proxy);
				result = pair.first;
				if (result != null) {
					JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
					if (jsonResult != null) {
						List<String> urlList = (List<String>) jsonResult.get("url");
						if (urlList != null) {
							for (String s : urlList) {
								if (s.contains(YUANCHUANG_FALG)) {
									ids.add(s);
								}
							}
						}
					}
				}

			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		LOG.info("total size=" + ids.size());
		return ids;
	}

	@SuppressWarnings("unchecked")
	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {

		NewsBaseTable table = new NewsBaseTable();
		String request = url;
		String result = "";
		Pair<String, byte[]> pair = fetResult(request, proxy);
		if (pair == null) {
			return new Pair<NewsBaseTable, byte[]>(table, "".getBytes());
		}
		result = pair.first;
		byte[] html = pair.second;
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {

				List<String> paraList = (List<String>) jsonResult.get("para_list");
				List<String> images = (List<String>) jsonResult.get("pic_list");
				List<String> keywords = (List<String>) jsonResult.get("keywords");
				String title = (String) jsonResult.get("title");
				String date = (String) jsonResult.get("date");
				if (validateObject(keywords)) {
					table.setTagName(keywords);
				}
				if (!StringUtil.isNullOrEmpty(title)) {
					table.setTitle(title);
				}
				String source = (String) jsonResult.get("source");
				if (validateObject(source)) {
					table.setSource(source);
				}
				if (!StringUtil.isNullOrEmpty(date)) {
					table.setPublishDate(genDate(date, "yyyy-MM-dd HH:mm:ss"));
				}

				if (validateObject(images)) {
					table = processImages(url, table, images);
				}
				if (paraList != null) {
					table.setContent(genContent(paraList, table.getImages()));
				}
				// set default value
				table.setId(StringUtil.genMD5Val(url));
				table.setCrawlUrl(url);
				table.setCrawlDate(new Date());
				table.setCrawlFlag(1);
			}
		}
		// save2hbase(table, html);
		return new Pair<NewsBaseTable, byte[]>(table, html);
	}

	// 原创第一页新闻
	@SuppressWarnings("unchecked")
	@Override
	protected List<String> genNewSeeds() {
		LOG.info("--- gennewSeeds of CaiJing YuanChuang ----");
		List<String> ids = new ArrayList<>();

		String url = String.format(NEWS_LIST_FORMAT, 1 + "");

		String result = "";
		Pair<String, byte[]> pair = fetResult(url, proxy);
		result = pair.first;
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {
				List<String> urlList = (List<String>) jsonResult.get("url");
				if (urlList != null) {
					for (String s : urlList) {
						if (s.contains(YUANCHUANG_FALG)) {
							ids.add(s);
						}
					}
				}
			}
		}

		LOG.info("total size=" + ids.size());
		return ids;
	}

}
